{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n",
      "MNIST ready\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('data/', one_hot=True)\n",
    "trainimg   = mnist.train.images\n",
    "trainlabel = mnist.train.labels\n",
    "testimg    = mnist.test.images\n",
    "testlabel  = mnist.test.labels\n",
    "print (\"MNIST ready\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#28*28 图片，\n",
    "n_input  = 784\n",
    "#输出的大小\n",
    "n_output = 10  # one_hot  5    0000010000\n",
    "# 权重\n",
    "# \n",
    "weights  = {\n",
    "        'wc1': tf.Variable(tf.random_normal([5, 5, 1, 20], stddev=0.1)),\n",
    "        'wc2': tf.Variable(tf.random_normal([5, 5, 20, 50], stddev=0.1)),\n",
    "        'wd1': tf.Variable(tf.random_normal([7*7*50, 500], stddev=0.1)),\n",
    "        'wd2': tf.Variable(tf.random_normal([500, n_output], stddev=0.1))\n",
    "    }\n",
    "# 表数字 i 类的偏置量\n",
    "biases   = {'bc1': tf.Variable(tf.random_normal([20], stddev=0.1)),\n",
    "                      'bc2': tf.Variable(tf.random_normal([50], stddev=0.1)),\n",
    "                      'bd1': tf.Variable(tf.random_normal([500], stddev=0.1)),\n",
    "                      'bd2': tf.Variable(tf.random_normal([n_output], stddev=0.1))}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN READY\n"
     ]
    }
   ],
   "source": [
    "def conv_basic(_input, _w, _b, _keepratio):\n",
    "        # [55000,784]\n",
    "        # INPUT，转换矩阵形状，改成一个28*28*1的，厚度自动\n",
    "        _input_r = tf.reshape(_input, shape=[-1, 28, 28, 1])\n",
    "        # CONV LAYER 1\n",
    "        #tf.nn.conv2d是TensorFlow里面实现卷积的函数\n",
    "        #tf.nn.conv2d(input, filter, strides, padding, use_cudnn_on_gpu=None, name=None)\n",
    "        #除去name参数用以指定该操作的name，与方法有关的一共五个参数：\n",
    "        #第一个参数input：指需要做卷积的输入图像，它要求是一个Tensor，具有[batch, in_height, in_width, in_channels]这样的shape，具体含义是[训练时一个batch的图片数量, 图片高度, 图片宽度, 图像通道数]，注意这是一个4维的Tensor，要求类型为float32和float64其中之一\n",
    "        #第二个参数filter：相当于CNN中的卷积核，它要求是一个Tensor，具有[filter_height, filter_width, in_channels, out_channels]这样的shape，具体含义是[卷积核的高度，卷积核的宽度，图像通道数，卷积核个数]，要求类型与参数input相同，有一个地方需要注意，第三维in_channels，就是参数input的第四维\n",
    "        #第三个参数strides：卷积时在图像每一维的步长，这是一个一维的向量，长度4\n",
    "        #第四个参数padding：string类型的量，只能是\"SAME\",\"VALID\"其中之一，当其为‘SAME’时，表示卷积核可以停留在图像边缘\n",
    "        #第五个参数：use_cudnn_on_gpu:bool类型，是否使用cudnn加速，默认为true\n",
    "        #结果返回一个Tensor，这个输出，就是我们常说的feature map\n",
    "        _conv1 = tf.nn.conv2d(_input_r, _w['wc1'], strides=[1, 1, 1, 1], padding='SAME')\n",
    "        #tf.nn.relu：修正线性，max(features, 0)\n",
    "        #tf.nn.bias_add:这个函数的作用是将偏差项 bias 加到 value 上面。\n",
    "        #这个操作你可以看做是 tf.add 的一个特例，其中 bias 必须是一维的。\n",
    "        #该API支持广播形式，因此 value 可以有任何维度。\n",
    "        #但是，该API又不像 tf.add ，可以让 bias 的维度和 value 的最后一维不同。\n",
    "        _conv1 = tf.nn.relu(tf.nn.bias_add(_conv1, _b['bc1']))\n",
    "        #最大池化\n",
    "        #value: 一个四维的Tensor。数据维度是 [batch, height, width, channels]。数据类型是float32，float64，qint8，quint8，qint32。\n",
    "        #ksize: 一个长度不小于4的整型数组。每一位上面的值对应于输入数据张量中每一维的窗口对应值。\n",
    "        #strides: 一个长度不小于4的整型数组。该参数指定滑动窗口在输入数据张量每一维上面的步长。\n",
    "        #padding: 一个字符串，取值为 SAME 或者 VALID 。\n",
    "        #name: （可选）为这个操作取一个名字。\n",
    "        _pool1 = tf.nn.max_pool(_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        #处理过拟合操作\n",
    "        _pool_dr1 = tf.nn.dropout(_pool1, _keepratio)\n",
    "        \n",
    "        # CONV LAYER 2\n",
    "        _conv2 = tf.nn.conv2d(_pool_dr1, _w['wc2'], strides=[1, 1, 1, 1], padding='SAME')\n",
    "        _conv2 = tf.nn.relu(tf.nn.bias_add(_conv2, _b['bc2']))\n",
    "        _pool2 = tf.nn.max_pool(_conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        _pool_dr2 = tf.nn.dropout(_pool2, _keepratio)\n",
    "        \n",
    "        \n",
    "        # VECTORIZE 向量化\n",
    "        _dense1 = tf.reshape(_pool_dr2, [-1, _w['wd1'].get_shape().as_list()[0]])\n",
    "        \n",
    "        # FULLY CONNECTED LAYER 1\n",
    "        _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1']))\n",
    "        _fc_dr1 = tf.nn.dropout(_fc1, _keepratio)\n",
    "        # FULLY CONNECTED LAYER 2\n",
    "        _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2'])\n",
    "        # RETURN\n",
    "        out = { 'input_r': _input_r, 'conv1': _conv1, 'pool1': _pool1, 'pool1_dr1': _pool_dr1,\n",
    "            'conv2': _conv2, 'pool2': _pool2, 'pool_dr2': _pool_dr2, 'dense1': _dense1,\n",
    "            'fc1': _fc1, 'fc_dr1': _fc_dr1, 'out': _out\n",
    "        }\n",
    "        return out\n",
    "print (\"CNN READY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#tf.random_normal,给出均值为mean，标准差为stdev的高斯随机数（场）\n",
    "a = tf.Variable(tf.random_normal([3, 3, 1, 64], stddev=0.1))\n",
    "print (a)\n",
    "a = tf.Print(a, [a], \"a: \")\n",
    "#Variable的初始化\n",
    "init = tf.global_variables_initializer()\n",
    "#建立会话\n",
    "sess = tf.Session()\n",
    "#执行初始化\n",
    "sess.run(init)\n",
    "sess.run(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print (help(tf.nn.conv2d))\n",
    "# print (help(tf.nn.max_pool))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GRAPH READY\n"
     ]
    }
   ],
   "source": [
    "#通过操作符号变量来描述这些可交互的操作单元\n",
    "#x不是一个特定的值，而是一个占位符placeholder，我们在TensorFlow运行计算时输入这个值。\n",
    "#我们希望能够输入任意数量的MNIST图像，每一张图展平成784维的向量。\n",
    "#我们用2维的浮点数张量来表示这些图，这个张量的形状是[None，784 ]。（这里的None表示此张量的第一个维度可以是任何长度的。）\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "keepratio = tf.placeholder(tf.float32)\n",
    "\n",
    "# FUNCTIONS\n",
    "\n",
    "#调用CNN函数，返回运算完的结果\n",
    "_pred = conv_basic(x, weights, biases, keepratio)['out']\n",
    "#交叉熵\n",
    "#首先看输入logits，它的shape是[batch_size, num_classes] ，\n",
    "#一般来讲，就是神经网络最后一层的输出z。\n",
    "#另外一个输入是labels，它的shape也是[batch_size, num_classes]，就是我们神经网络期望的输出。\n",
    "#这个函数的作用就是计算最后一层是softmax层的cross entropy，只不过tensorflow把softmax计算与cross entropy计算放到一起了。\n",
    "#用一个函数来实现，用来提高程序的运行速度\n",
    "#http://www.jianshu.com/p/fb119d0ff6a6\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=_pred,labels=y))\n",
    "#Adam算法\n",
    "#。AdamOptimizer通过使用动量（参数的移动平均数）来改善传统梯度下降，促进超参数动态调整。\n",
    "#我们可以通过创建标签错误率的摘要标量来跟踪丢失和错误率\n",
    "#一个寻找全局最优点的优化算法，引入了二次方梯度校正。\n",
    "#相比于基础SGD算法，1.不容易陷于局部优点。2.速度更快\n",
    "optm = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)\n",
    "#比较\n",
    "_corr = tf.equal(tf.argmax(_pred,1), tf.argmax(y,1)) \n",
    "#cast:将x或者x.values转换为dtype\n",
    "#tf.reduce_mean  求tensor中平均值\n",
    "#http://blog.csdn.net/lenbow/article/details/52152766\n",
    "accr = tf.reduce_mean(tf.cast(_corr, tf.float32)) \n",
    "# 初始化\n",
    "init = tf.global_variables_initializer()\n",
    "init_local = tf.local_variables_initializer() \n",
    "# SAVER\n",
    "print (\"GRAPH READY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/015 cost: 0.120625865\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.987\n",
      "Epoch: 001/015 cost: 0.034912461\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.988\n",
      "Epoch: 002/015 cost: 0.026420425\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.989\n",
      "Epoch: 003/015 cost: 0.020407431\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.993\n",
      "Epoch: 004/015 cost: 0.016653247\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.990\n",
      "Epoch: 005/015 cost: 0.013498855\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.993\n",
      "Epoch: 006/015 cost: 0.011933199\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.993\n",
      "Epoch: 007/015 cost: 0.010569925\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.994\n",
      "Epoch: 008/015 cost: 0.008824048\n",
      " Training accuracy: 0.938\n",
      " Test accuracy: 0.993\n",
      "Epoch: 009/015 cost: 0.008367225\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.994\n",
      "Epoch: 010/015 cost: 0.007780030\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.992\n",
      "Epoch: 011/015 cost: 0.007406353\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.994\n",
      "Epoch: 012/015 cost: 0.006567118\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.993\n",
      "Epoch: 013/015 cost: 0.006129434\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.992\n",
      "Epoch: 014/015 cost: 0.006150965\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.994\n",
      "OPTIMIZATION FINISHED\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "sess.run(init_local)\n",
    "#训练次数\n",
    "training_epochs = 15\n",
    "#batch\n",
    "batch_size      = 5\n",
    "#执行到第几次显示运行结果\n",
    "display_step    = 10\n",
    "for epoch in range(training_epochs):\n",
    "    #平均误差\n",
    "    avg_cost = 0.\n",
    "    total_batch = int(mnist.train.num_examples/batch_size)\n",
    "    #total_batch = 10\n",
    "    # Loop over all batches  循环所有批次\n",
    "    list1=[]\n",
    "    for i in range(total_batch):\n",
    "        #去除训练集合的下10条\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        # Fit training using batch data 使用批处理数据进行培训\n",
    "        sess.run(optm, feed_dict={x: batch_xs, y: batch_ys, keepratio:0.7})\n",
    "        # Compute average loss  计算平均损失\n",
    "        avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})/total_batch\n",
    "        train_acc = sess.run(accr, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})\n",
    "        \n",
    "    # Display logs per epoch step 显示现在的状态\n",
    "    if epoch % display_step == 0: \n",
    "        print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch, training_epochs, avg_cost))\n",
    "        train_acc = sess.run(accr, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})\n",
    "        \n",
    "        print (\" Training accuracy: %.3f\" % (train_acc))\n",
    "        test_acc = sess.run(accr, feed_dict={x: testimg, y: testlabel, keepratio:1.})\n",
    "        print (\" Test accuracy: %.3f\" % (test_acc))\n",
    "\n",
    "print (\"OPTIMIZATION FINISHED\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#1.数据载入+预处理\n",
    "#2.神经网络结构设计\n",
    "#3.选择损失函数\n",
    "#4.选择SGD优化函数、设置正确率计算方法\n",
    "#5.设置迭代次数、每次训练的数据量\n",
    "#6.执行迭代训练（设置每多少次，看一下当前状态）"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
